package test

import net.ruippeixotog.scalascraper.browser.JsoupBrowser

import java.io.File
import net.ruippeixotog.scalascraper.dsl.DSL._
import net.ruippeixotog.scalascraper.dsl.DSL.Extract._
import net.ruippeixotog.scalascraper.dsl.DSL.Parse._
import org.jsoup.Jsoup
import play.api.libs.json.{JsArray, JsNull, JsString, JsValue, Json}

import scala.jdk.CollectionConverters._
import implicits.Implicits._
import net.ruippeixotog.scalascraper.model.Element

import scala.reflect.ClassTag

/**
 * Created by yz on 13/12/2021
 */
object ParseHtml {

  def main(args: Array[String]): Unit = {

    val parentDir = new File("E:\\snp_database\\test")
    val rsDir = new File(parentDir, "rs")
    rsDir.myListFiles.filter { dir =>
      dir.getName.startsWith("rs")
    }.filter { dir =>
      val jsonFile = new File(dir, s"${dir.getName}.json")
//      !jsonFile.exists() && dir.getName != "rs20607903"
//      dir.getName == "rs6564851"
      true
    }.map { dir =>
//      println(dir.getName)
      val jsonFile = new File(dir, s"${dir.getName}.json")
      val htmlFile = new File(dir, s"${dir.getName}.html")
      val doc = JsoupBrowser().parseFile(htmlFile)
      val summaryE = doc >> element(".summary-box")
      val keyEs = summaryE >> elements("dt")
      val valueEs = summaryE >> elements("dd")
      val summaryMap = keyEs.zip(valueEs).map { case (ke, ve) =>
        val k = ke >> text
        val v = k match {
          case "Position" =>
            val vStr = ve >> elements("span:not(.usa-sr-only)") >> allText
            JsString(vStr)
          case "Frequency" =>
            val es = (ve >> elementList(":not(dd,#expandfrequency,div#remn_summ_freq)")).filter { e =>
              (e >?> element("#expandfrequency")).isEmpty
            }
            val vs = es >> allText
            Json.toJson(vs)
          case _ => val vStr = ve >> text
            JsString(vStr)
        }
        (k -> v)
      }.toList.toSeqMap
      val summaryJson = Json.toJson(summaryMap)
      val detailsE = doc >> element("table#genomics_placements_table")
      val detailHeaders = detailsE >> elements("th") >> texts
      val detailRows = (detailsE >> elementList("tbody tr")).map { e =>
        val tds = e >> elements("td") >> texts
        detailHeaders.zip(tds).toList.toSeqMap
      }
      val detailJson = Json.toJson(detailRows)
      val geneDetailE = doc >?> element("div#trans_anno_allele table")
      val geneDetailJson = getRowsOp(geneDetailE)
      val significancesE = doc >?> element("table#clinical_significance_datatable")
      val significancesJson = getRowsOp(significancesE)
      val alfaFrequencyE = doc >> element("div#popfreq_table table")
      val alfaJson = getRows(alfaFrequencyE)
      val frequencyE = doc >> element("div#dbsnp_freq_table table")
      val frequencyJson = getRows(frequencyE)
      val hgvsE = doc >> element("div#hgvs table")
      val hgvsJson = getRows(hgvsE)
      val submissionsE = doc >> element("div#submission table")
      val submissionJson = getRows(submissionsE)
      val historyE = doc >?> element("table#up_hist_datatable")
      val historyJson = getRowsOp(historyE)
      val addedHistoryE = doc >> element("div#obs_present table")
      val addedHistoryJson = getRows(addedHistoryE)
      val publicationE = doc >?> element("div#publication table")
      if (publicationE.isEmpty) {
        println(dir.getName)
      }
      val publicationJson = getRowsOp(publicationE)
      val json = Json.obj("summary" -> summaryJson, "detail" -> detailJson, "geneDetail" -> geneDetailJson,
        "significances" -> significancesJson, "alfa" -> alfaJson, "frequency" -> frequencyJson, "hgvs" -> hgvsJson,
        "submission" -> submissionJson, "history" -> historyJson, "addedHistory" -> addedHistoryJson,
        "publication" -> publicationJson)
//      Json.prettyPrint(json).toFile(jsonFile)
    }.distinct.foreach(println(_))

    def getRowsOp[X: ClassTag](tableEOp: Option[Element]): JsValue = {
      tableEOp.map { tableE =>
        val headers = tableE >> elements("th") >> texts
        val rows = (tableE >> elementList("tbody tr")).map { e =>
          val tds = e >> elements("td") >> texts
          headers.zip(tds).toList.toSeqMap
        }.filterNot { map =>
          map.valueList.forall(_.isBlank)
        }
        Json.toJson(rows)
      }.getOrElse {
        JsNull
      }
    }

    def getRows(tableE: Element): JsValue = {
      getRowsOp(Some(tableE))
    }


  }

}
